Analyze AFM tip solvation¶
This notebook demonstrates deposition of an SDS adsorption layer on a non-spherical AFM tip model.
Initialization¶
IPython magic¶
In [13]:
%load_ext autoreload
%autoreload 2
In [14]:
%aimport
Imports¶
In [15]:
import ase.io # here used for reading pdb files
from ase.visualize import view
from ase.visualize.plot import plot_atoms # has nasty offset issues
from cycler import cycler # here used for cycling through colors in plots
import datetime
import fabric # for pythonic ssh connections
from fireworks import LaunchPad, Firework, Tracker, Workflow
from fireworks import FileTransferTask, PyTask, ScriptTask
# FireWorks functionality
from fireworks import Firework, LaunchPad, ScriptTask, Workflow
from fireworks.user_objects.firetasks.templatewriter_task import TemplateWriterTask
from fireworks.user_objects.firetasks.filepad_tasks import AddFilesTask, GetFilesTask, GetFilesByQueryTask
from imteksimfw.fireworks.user_objects.firetasks.cmd_tasks import CmdTask
from fireworks.utilities.filepad import FilePad # direct FilePad access, similar to the familiar LaunchPad
from collections.abc import Iterable
import glob
import gc # manually clean up memory with gc.collect()
import gromacs # GromacsWrapper, here used for evoking gmc commands, reading and writing .ndx files
# from io import StringIO, TextIOWrapper
import io
from IPython.display import display, Image #, Video # display image files within notebook
from ipywidgets import Video # display video within notebook
import itertools # for products of iterables
import json # generic serialization of lists and dicts
import jinja2 # here used for filling packmol input script template
import jinja2.meta # for gathering variables in a jinja2 template
import logging
import matplotlib.pyplot as plt
import MDAnalysis as mda # here used for reading and analyzing gromacs trajectories
import MDAnalysis.analysis.rdf as mda_rdf
import MDAnalysis.analysis.rms as mda_rms
from mpl_toolkits.mplot3d import Axes3D # here used for 3d point cloud scatter plot
import miniball # finds minimum bounding sphere of a point set
import nglview
import numpy as np
import os, os.path
import pandas as pd
import panedr # reads GROMACS edr into pandas df, requires pandas and pbr
import parmed as pmd # has quite a few advantages over ASE when it comes to parsing pdb
from pprint import pprint
import pymongo # for sorting in queries
import scipy.constants as sc
import subprocess # used for evoking external packmol
import sys
import tempfile
import yaml
GromacsWrapper might need a file ~/.gromacswrapper.cfg with content
[Gromacs]
tools = gmx gmx_d
# gmx_mpi_d gmx_mpi_d
# name of the logfile that is written to the current directory
logfilename = gromacs.log
# loglevels (see Python's logging module for details)
# ERROR only fatal errors
# WARN only warnings
# INFO interesting messages
# DEBUG everything
# console messages written to screen
loglevel_console = INFO
# file messages written to logfilename
loglevel_file = DEBUG
in order to know the GROMACS executables it is allowed to use. Otherwise,
calls to gmx_mpi or gmx_mpi_d without MPI wrapper might lead to MPI
warnings in output that cause GromacsWrapper to fail.
Logging¶
In [16]:
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
ParmEd needs to know the GROMACS topology folder, usually get this from
envionment variable GMXLIB:
Function definitions¶
In [17]:
def find_undeclared_variables(infile):
"""identify all variables evaluated in a jinja 2 template file"""
env = jinja2.Environment()
with open(infile) as template_file:
parsed = env.parse(template_file.read())
undefined = jinja2.meta.find_undeclared_variables(parsed)
return undefined
In [18]:
def memuse():
"""Quick overview on memory usage of objects in Jupyter notebook"""
# https://stackoverflow.com/questions/40993626/list-memory-usage-in-ipython-and-jupyter
# These are the usual ipython objects, including this one you are creating
ipython_vars = ['In', 'Out', 'exit', 'quit', 'get_ipython', 'ipython_vars']
# Get a sorted list of the objects and their sizes
return sorted([(x, sys.getsizeof(globals().get(x))) for x in dir(sys.modules['__main__']) if not x.startswith('_') and x not in sys.modules and x not in ipython_vars], key=lambda x: x[1], reverse=True)
Global settings¶
In [19]:
# pandas settings
pd.options.display.max_rows = 200
pd.options.display.max_columns = 16
pd.options.display.max_colwidth = 256
In [20]:
os.environ['GMXLIB'] = '/gmx_top'
In [21]:
# pmd.gromacs.GROMACS_TOPDIR = os.environ['GMXLIB']
pmd.gromacs.GROMACS_TOPDIR = '/gmx_top'
In [22]:
# prefix = '/mnt/dat/work/testuser/indenter/sandbox/20191110_packmol'
prefix = '/mnt/dat/work'
In [23]:
work_prefix = '/mnt/dat/work/tmp'
In [24]:
try:
os.mkdir(work_prefix)
except FileExistsError as exc:
print(exc)
In [25]:
os.chdir(work_prefix)
In [26]:
# the FireWorks LaunchPad
lp = LaunchPad.auto_load() #Define the server and database
# FilePad behaves analogous to LaunchPad
fp = FilePad.auto_load()
Conversion from LAMMPS data format to PDB¶
The following bash / tcl snippet converts a LAMMPS data file to PDB, assigning the desired names as mapped in a yaml file
#!/bin/bash
# echo "package require jlhvmd; jlh lmp2pdb indenter.lammps indenter.pdb" | vmd -eofexit
vmd -eofexit << 'EOF'
package require jlhvmd
topo readlammpsdata indenter.lammps
jlh type2name SDS_type2name.yaml
jlh name2res SDS_name2res.yaml
set sel [atomselect top all]
$sel writepdb indenter.pdb
EOF
pdb_chain.py indenter.pdb > indenter_wo_chainid.pdb
pdb_reres_by_atom_9999.py indenter_wo_chainid.pdb > indenter_reres.pdb
Requires
- VMD (tested with 1.9.3) with topotools
- jlhvmd VMD plugin: https://github.com/jotelha/jlhvmd
- pdb-tools: https://github.com/haddocking/pdb-tools/
Overview¶
Overview on projects in database¶
In [27]:
query = {'metadata.datetime': {'$gt': '2020'} }
In [28]:
fp.filepad.count_documents(query)
Out[28]:
In [29]:
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": { 'project': '$metadata.project' },
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # pull 'project' field up in hierarchy
"$addFields": {
"project": "$_id.project",
},
},
{ # drop nested '_id.project'
"$project": {
"_id": False
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
# sort_aggregation
#aggregation_pipeline = [ match_aggregation, group_aggregation, set_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [c for c in cursor]
res_df = pd.DataFrame(data=res) # pandas Dataframe is just nice for printing in notebook
In [30]:
res_df
Out[30]:
Overview on steps in project¶
In [31]:
project_id = '2020-07-19-passivation'
In [32]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
}
In [33]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[33]:
In [34]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'step': '$metadata.step',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['step', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [35]:
res_df
Out[35]:
In [36]:
res_df['step'].values
Out[36]:
Overview on objects in project¶
In [37]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
}
In [38]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[38]:
In [39]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
'step': '$metadata.step',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'step', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [40]:
res_df
Out[40]:
Overview on images by distinct steps¶
In [41]:
query = {
'metadata.project': project_id,
'metadata.type': 'png_file',
}
In [42]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[42]:
In [43]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
'step': '$metadata.step',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['step', 'type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [44]:
res_df
Out[44]:
In [45]:
res_df["step"][0]
Out[45]:
Packing visualization¶
Indenter bounding sphere¶
In [46]:
query = {
'metadata.project': project_id,
'metadata.type': 'png_file',
'metadata.step': {'$regex': 'IndenterBoundingSphere'}
}
In [47]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[47]:
In [48]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["gfs_id"])
with tempfile.NamedTemporaryFile(suffix='.png') as tmp:
tmp.write(content)
obj_list.append(Image(filename=tmp.name))
print('.',end='')
In [49]:
for obj in obj_list:
display(obj)
Surfactant measures¶
In [50]:
query = {
'metadata.project': project_id,
'metadata.type': 'png_file',
'metadata.step': {'$regex': 'SurfactantMoleculeMeasures'}
}
In [51]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[51]:
In [52]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["gfs_id"])
with tempfile.NamedTemporaryFile(suffix='.png') as tmp:
tmp.write(content)
obj_list.append(Image(filename=tmp.name))
print('.',end='')
In [53]:
obj_list[0]
Out[53]:
Packing constraints¶
In [54]:
query = {
'metadata.project': project_id,
'metadata.type': 'png_file',
'metadata.step': {'$regex': 'PackingConstraintSpheres'}
}
In [55]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[55]:
In [56]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["gfs_id"])
with tempfile.NamedTemporaryFile(suffix='.png') as tmp:
tmp.write(content)
obj_list.append(Image(filename=tmp.name))
print('.',end='')
In [57]:
obj_list[0]
Out[57]:
Packed film¶
In [58]:
query = {
'metadata.project': project_id,
'metadata.type': 'png_file',
'metadata.step': {'$regex': 'SphericalSurfactantPacking'}
}
In [59]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[59]:
In [60]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["gfs_id"])
with tempfile.NamedTemporaryFile(suffix='.png') as tmp:
tmp.write(content)
obj_list.append(Image(filename=tmp.name))
print('.',end='')
In [61]:
for obj in obj_list:
display(obj)
Energy minimization analysis¶
Overview on objects in step¶
In [62]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsEnergyMinimization:ProcessAnalyzeAndVisualize:push_filepad'
}
In [63]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[63]:
In [64]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [65]:
res_df
Out[65]:
Global observables¶
In [66]:
query = {
"metadata.project": project_id,
'metadata.step': 'GromacsEnergyMinimization:ProcessAnalyzeAndVisualize:push_filepad', #{'$regex': 'GromacsEnergyMinimization'}
"metadata.type": 'energy_file',
}
fp.filepad.count_documents(query)
Out[66]:
In [67]:
parameter_dict = {'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [68]:
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
In [69]:
[ c for c in cursor]
Out[69]:
In [70]:
res_mi_list = []
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
with tempfile.NamedTemporaryFile(suffix='.edr') as tmp:
tmp.write(content)
em_df = panedr.edr_to_df(tmp.name)
mi = pd.MultiIndex.from_product(
[c["_id"].values(),em_df.index],
names=[*c["_id"].keys(),'step'])
em_mi_df = em_df.set_index(mi)
res_mi_list.append(em_mi_df)
print('.',end='')
print('')
res_mi_df = pd.concat(res_mi_list)
res_df = res_mi_df.reset_index()
In [71]:
res_mi_df
Out[71]:
In [72]:
res_df
Out[72]:
In [73]:
y_quantities = [
'Potential',
'Pressure',
'Bond',
'Coulomb (SR)',
'Coul. recip.',
]
positions = [
(0,0),
(0,1),
(1,0),
(2,0),
(2,1),
]
fig, ax = plt.subplots(3,2,figsize=(10,12))
for key, grp in res_df.groupby(['nmolecules']):
for y_quantity, position in zip(y_quantities, positions):
grp.plot('Time',y_quantity,ax=ax[position],label=key,title=y_quantity)
fig.tight_layout()
Visualize trajectory¶
In [74]:
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsEnergyMinimization:ProcessAnalyzeAndVisualize:push_filepad',
'metadata.type': 'mp4_file',
}
In [75]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[75]:
In [76]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
'step': '$metadata.step',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['step', 'type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [77]:
res_df
Out[77]:
In [78]:
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["gfs_id"])
with tempfile.NamedTemporaryFile(suffix='.mp4',delete=False) as tmp:
tmp.write(content)
# obj_list.append(Video(filename=tmp.name))
# obj_list.append(tmp.name)
obj_list.append(Video.from_file(tmp.name))
print('.',end='')
In [79]:
obj_list[-1]
Pulling analysis¶
Overview on objects in step¶
In [80]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsPull:ProcessAnalyzeAndVisualize:push_filepad'
}
In [81]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[81]:
In [82]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [83]:
res_df
Out[83]:
Global observables¶
The gmx energy table:
1 Restraint-Pot. 2 U-B 3 Proper-Dih. 4 LJ-14
5 Coulomb-14 6 LJ-(SR) 7 Coulomb-(SR) 8 Coul.-recip.
9 Position-Rest. 10 COM-Pull-En. 11 Potential 12 Kinetic-En.
13 Total-Energy 14 Temperature 15 Pressure 16 Constr.-rmsd
17 Vir-XX 18 Vir-XY 19 Vir-XZ 20 Vir-YX
21 Vir-YY 22 Vir-YZ 23 Vir-ZX 24 Vir-ZY
25 Vir-ZZ 26 Pres-XX 27 Pres-XY 28 Pres-XZ
29 Pres-YX 30 Pres-YY 31 Pres-YZ 32 Pres-ZX
33 Pres-ZY 34 Pres-ZZ 35 #Surf*SurfTen 36 T-restconverted to dict with regex
\s+([0-9]+)\s+([^\s]+)
and replacement
'$2': $1,\nIn [84]:
gmx_energy_dict = {
'Restraint-Pot.': 1,
'U-B': 2,
'Proper-Dih.': 3,
'LJ-14': 4,
'Coulomb-14': 5,
'LJ-(SR)': 6,
'Coulomb-(SR)': 7,
'Coul.-recip.': 8,
'Position-Rest.': 9,
'COM-Pull-En.': 10,
'Potential': 11,
'Kinetic-En.': 12,
'Total-Energy': 13,
'Temperature': 14,
'Pressure': 15,
'Constr.-rmsd': 16,
'Vir-XX': 17,
'Vir-XY': 18,
'Vir-XZ': 19,
'Vir-YX': 20,
'Vir-YY': 21,
'Vir-YZ': 22,
'Vir-ZX': 23,
'Vir-ZY': 24,
'Vir-ZZ': 25,
'Pres-XX': 26,
'Pres-XY': 27,
'Pres-XZ': 28,
'Pres-YX': 29,
'Pres-YY': 30,
'Pres-YZ': 31,
'Pres-ZX': 32,
'Pres-ZY': 33,
'Pres-ZZ': 34,
'#Surf*SurfTen': 35,
'T-rest': 36,
}
In [85]:
query = {
"metadata.project": project_id,
"metadata.type": 'energy_file',
"metadata.step": "GromacsPull:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[85]:
In [86]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [87]:
gmx_energy_selection = [
'Restraint-Pot.',
'Position-Rest.',
'COM-Pull-En.',
'Potential',
'Kinetic-En.',
'Total-Energy',
'Temperature',
'Pressure',
'Constr.-rmsd',
]
In [88]:
res_list = []
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
aggregation_pipeline = [ match_aggregation, sort_aggregation, group_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res_df_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
#df = panedr.edr_to_df(tmp.name), fails
tmpin = tempfile.NamedTemporaryFile(mode='w+b',suffix='.edr', delete=False)
# cur_res_dict = {}
with tmpin:
tmpin.write(content)
#tmpin.seek(0)
res_df = None
for sel in gmx_energy_selection:
try:
tmpout = tempfile.NamedTemporaryFile(suffix='.xvg', delete=False)
res = gromacs.energy(f=tmpin.name,o=tmpout.name,
input=str(gmx_energy_dict[sel]))
#with open(tmpout.name,'r') as f:
# xvg = f.read()
#tmpout.delete()
xvg = mda.auxiliary.XVG.XVGReader(tmpout.name)
xvg_time = xvg.read_all_times()
xvg_data = np.array([ f.data[1:] for f in xvg ]).flatten() # 1st entry contains times
os.unlink(tmpout.name)
except:
logger.warning("Failed to read '{:s}' from data set {:d}.".format(sel,i))
failed_list.append((nmolecules, sel))
else:
r = {'nmolecules': [nmolecules]*len(xvg_time), 'time': xvg_time, sel: xvg_data}
cur_df = pd.DataFrame(r)
if res_df is None:
res_df = cur_df
else:
res_df = pd.merge(res_df, cur_df, how='outer', on=['nmolecules', 'time'])
res_df_list.append(res_df)
os.unlink(tmpin.name)
print('.',end='')
print('')
res_df = pd.concat(res_df_list)
res_df_mi = res_df.set_index(['nmolecules','time'])
In [89]:
res_df_mi
Out[89]:
In [90]:
cols = 2
y_quantities = [
'Restraint-Pot.',
'Position-Rest.',
'COM-Pull-En.',
'Potential',
'Kinetic-En.',
'Total-Energy',
'Temperature',
'Pressure',
'Constr.-rmsd',
]
n = len(y_quantities)
rows = round(n/cols)
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
for key, grp in res_df.groupby(['nmolecules']):
for y_quantity, position in zip(y_quantities, positions):
grp.plot('time',y_quantity,ax=ax[position],label=key,title=y_quantity)
fig.tight_layout()
Pulling forces¶
In [91]:
res_df_list = []
failed_list = []
query = {
"metadata.project": project_id,
"metadata.step": "GromacsPull:ProcessAnalyzeAndVisualize:push_filepad",
"metadata.type": 'pullf_file',
}
fp.filepad.count_documents(query)
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
aggregation_pipeline = [ match_aggregation, sort_aggregation, group_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
tmpin = tempfile.NamedTemporaryFile(mode='w+b',suffix='.xvg', delete=False)
with tmpin:
tmpin.write(content)
try:
xvg = mda.auxiliary.XVG.XVGReader(tmpin.name)
xvg_time = xvg.read_all_times()
xvg_data = np.array([ f.data[1:] for f in xvg ])# .flatten() # 1st entry contains times
except:
logger.warning("Failed to read data set {:d}.".format(i))
failed_list.append(nmolecules)
else:
res_df_list.append(pd.DataFrame({
'nmolecules': np.array([nmolecules]*len(xvg_time), dtype=int),
'time': xvg_time,
**{i: xvg_data[:,i] for i in range(nmolecules)}
}))
os.unlink(tmpin.name)
print('.',end='')
print('')
res_df = pd.concat(res_df_list)
res_df_mi = res_df.set_index(['nmolecules','time'])
In [92]:
# pulling forces
n = len(res_df['nmolecules'].unique())
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
for pos, (key, grp) in zip(positions,res_df.groupby(['nmolecules'])):
columns = list(set(grp.columns) - set(['nmolecules','time']))
grp.plot('time', columns, ax=ax[pos],title=key,legend=None)
fig.tight_layout()
In [93]:
# mean pulling force
fig, ax = plt.subplots(1,1,figsize=(5,4))
for key, grp in res_df.groupby(['nmolecules']):
columns = list(set(grp.columns) - set(['nmolecules','time']))
grp = grp.set_index('time')
grp = grp.drop(columns='nmolecules')
grp.mean(axis=1).plot(legend=True, label=key, ax=ax)
fig.tight_layout()
#fig.legend()
Pulling groups movement¶
In [94]:
res_df_list = []
failed_list = []
query = {
"metadata.project": project_id,
"metadata.step": "GromacsPull:ProcessAnalyzeAndVisualize:push_filepad",
"metadata.type": 'pullx_file',
}
fp.filepad.count_documents(query)
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
aggregation_pipeline = [ match_aggregation, sort_aggregation, group_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
for i, c in enumerate(cursor):
print(c["_id"])
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"]) # int(metadata["metadata"]["nmolecules"])
tmpin = tempfile.NamedTemporaryFile(mode='w+b',suffix='.xvg', delete=False)
with tmpin:
tmpin.write(content)
try:
xvg = gromacs.fileformats.XVG(tmpin.name)
xvg_time = xvg.array[0,:]
#xvg_labels = ['1', '1 ref', '1 dX', '1 dY', '1 dZ', '1 g 1 X', '1 g 1 Y', '1 g 1 Z', '1 g 2 X', '1 g 2 Y', '1 g 2 Z']
N_pull_coords = nmolecules
N_cols = len(xvg.names)
N_cols_per_coord = int(N_cols / N_pull_coords)
xvg_labels = xvg.names[:N_cols_per_coord]
xvg_data = {}
for j in range(N_pull_coords):
for k in range(N_cols_per_coord):
xvg_data[(j,xvg_labels[k])] = xvg.array[
1+j*N_cols_per_coord+k,:]
except:
logger.exception("Failed to read data set {:d}.".format(i))
failed_list.append(nmolecules)
else:
res_df_list.append(pd.DataFrame({
'nmolecules': np.array([nmolecules]*len(xvg_time), dtype=int),
'time': xvg_time,
**xvg_data # {i: xvg_data[:,i] for i in range(nmolecules)}
}))
os.unlink(tmpin.name)
res_df = pd.concat(res_df_list)
res_df_mi = res_df.set_index(['nmolecules','time'])
res_df_mi.columns = pd.MultiIndex.from_tuples(res_df_mi.columns, names=['nmolecule', 'coord'])
In [95]:
res_df_mi
Out[95]:
In [96]:
res_df = res_df_mi.groupby(axis=1,level='coord').mean().reset_index()
In [97]:
res_df
Out[97]:
In [98]:
cols = 2
y_quantities = [
'1',
'1 ref',
'1 dX',
'1 dY',
'1 dZ',
'1 g 1 X',
'1 g 1 Y',
'1 g 1 Z',
'1 g 2 X',
'1 g 2 Y',
'1 g 2 Z'
]
n = len(y_quantities)
rows = round(n/cols)
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
for key, grp in res_df.groupby(['nmolecules']):
for y_quantity, position in zip(y_quantities, positions):
grp.plot('time',y_quantity,ax=ax[position],label=key,title=y_quantity)
fig.tight_layout()
Rendered videos¶
In [99]:
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsPull:ProcessAnalyzeAndVisualize:push_filepad',
'metadata.type': 'mp4_file',
}
In [100]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[100]:
In [101]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
'step': '$metadata.step',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['step', 'type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [102]:
res_df
Out[102]:
In [103]:
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["gfs_id"])
with tempfile.NamedTemporaryFile(suffix='.mp4',delete=False) as tmp:
tmp.write(content)
# obj_list.append(Video(filename=tmp.name))
# obj_list.append(tmp.name)
obj_list.append(Video.from_file(tmp.name))
print('.',end='')
In [104]:
for obj in obj_list:
display(obj)
Pre-evaluated RDF¶
Overview¶
In [105]:
query = {
"metadata.project": project_id,
"metadata.type": {'$regex': '.*rdf$'},
"metadata.step": "GromacsPull:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[105]:
In [106]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [107]:
res_df
Out[107]:
Substrate - surfactant head RDF¶
In [108]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [109]:
query = {
"metadata.project": project_id,
"metadata.type": 'substrate_surfactant_head_rdf',
"metadata.step": "GromacsPull:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[109]:
In [110]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
aggregation_pipeline = [ match_aggregation, sort_aggregation, group_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [111]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Substrate - surfactant tail RDF¶
In [112]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [113]:
query = {
"metadata.project": project_id,
"metadata.type": 'substrate_surfactant_tail_rdf',
"metadata.step": "GromacsPull:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[113]:
In [114]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [115]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Surfactant head - surfactant tail RDF¶
In [116]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [117]:
query = {
"metadata.project": project_id,
"metadata.type": 'surfactant_head_surfactant_tail_rdf',
"metadata.step": "GromacsPull:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[117]:
In [118]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [119]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Energy minimization after solvation analysis¶
Overview on objects in step¶
In [120]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsEnergyMinimizationAfterSolvation:ProcessAnalyzeAndVisualize:push_filepad'
}
In [121]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[121]:
In [122]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [123]:
res_df
Out[123]:
Global observables¶
In [124]:
query = {
"metadata.project": project_id,
'metadata.step': 'GromacsEnergyMinimizationAfterSolvation:ProcessAnalyzeAndVisualize:push_filepad', #{'$regex': 'GromacsEnergyMinimization'}
"metadata.type": 'energy_file',
}
fp.filepad.count_documents(query)
Out[124]:
In [125]:
parameter_dict = {'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [126]:
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
In [127]:
res_mi_list = []
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
with tempfile.NamedTemporaryFile(suffix='.edr') as tmp:
tmp.write(content)
em_df = panedr.edr_to_df(tmp.name)
mi = pd.MultiIndex.from_product(
[c["_id"].values(),em_df.index],
names=[*c["_id"].keys(),'step'])
em_mi_df = em_df.set_index(mi)
res_mi_list.append(em_mi_df)
print('.',end='')
print('')
res_mi_df = pd.concat(res_mi_list)
res_df = res_mi_df.reset_index()
In [128]:
res_mi_df
Out[128]:
In [129]:
y_quantities = [
'Potential',
'Pressure',
'Bond',
'Coulomb (SR)',
'Coul. recip.',
]
positions = [
(0,0),
(0,1),
(1,0),
(2,0),
(2,1),
]
fig, ax = plt.subplots(3,2,figsize=(10,12))
for key, grp in res_df.groupby(['nmolecules']):
for y_quantity, position in zip(y_quantities, positions):
grp.plot('Time',y_quantity,ax=ax[position],label=key,title=y_quantity)
fig.tight_layout()
Visualize trajectory¶
In [130]:
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsEnergyMinimizationAfterSolvation:ProcessAnalyzeAndVisualize:push_filepad',
'metadata.type': 'mp4_file',
}
In [131]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[131]:
In [132]:
# check
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'nmolecules': '$metadata.system.surfactant.nmolecules'
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# for i, c in enumerate(cursor):
# content, metadata = fp.get_file_by_id(c["latest"])
# nmolecules = int(c["_id"]["nmolecules"])
res = [ {**c['_id'], **c} for c in cursor]
columns = ['nmolecules', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [133]:
res_df
Out[133]:
In [134]:
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_dict = {}
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
# print(metadata['metadata'])
with tempfile.NamedTemporaryFile(suffix='.mp4',delete=False) as tmp:
tmp.write(content)
# obj_list.append(Video(filename=tmp.name))
# obj_list.append(tmp.name)
obj_dict.update({metadata['metadata']['system']['surfactant']['nmolecules']: Video.from_file(tmp.name)})
print('.',end='')
In [135]:
for key, obj in obj_dict.items():
print(key)
display(obj)
NVT equilibration analysis¶
Overview on objects in step¶
In [136]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsNVTEquilibration:ProcessAnalyzeAndVisualize:push_filepad'
}
In [137]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[137]:
In [138]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [139]:
res_df
Out[139]:
Global observables¶
In [140]:
query = {
"metadata.project": project_id,
'metadata.step': 'GromacsNVTEquilibration:ProcessAnalyzeAndVisualize:push_filepad',
"metadata.type": 'energy_file',
}
fp.filepad.count_documents(query)
Out[140]:
In [141]:
parameter_dict = {'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [142]:
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
In [143]:
[ c for c in cursor]
Out[143]:
In [144]:
res_list = []
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
with tempfile.NamedTemporaryFile(suffix='.edr') as tmp:
tmp.write(content)
res_df = panedr.edr_to_df(tmp.name)
mi = pd.MultiIndex.from_product(
[c["_id"].values(),res_df.index],
names=[*c["_id"].keys(),'step'])
res_mi_df = res_df.set_index(mi)
res_list.append(res_mi_df)
print('.',end='')
print('')
res_df_mi = pd.concat(res_list)
res_df = res_df_mi.reset_index()
In [145]:
res_df.columns
Out[145]:
In [146]:
res_df_mi
Out[146]:
In [147]:
#n = len(res_df['nmolecules'].unique())
y_quantities = [
'Temperature',
'Pressure',
'Potential',
'Bond',
'Coulomb (SR)',
'Coul. recip.',
]
n = len(y_quantities)
cols = 2
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
for key, grp in res_df.groupby(['nmolecules']):
for y_quantity, position in zip(y_quantities, positions):
grp.plot('Time',y_quantity,ax=ax[position],label=key,title=y_quantity)
fig.tight_layout()
Visualize trajectory¶
In [148]:
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsNVTEquilibration:ProcessAnalyzeAndVisualize:push_filepad',
'metadata.type': 'mp4_file',
}
In [149]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[149]:
In [150]:
parameter_dict = {'nmolecules': 'metadata.system.surfactant.nmolecules'}
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_dict = {}
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
# print(metadata['metadata'])
with tempfile.NamedTemporaryFile(suffix='.mp4',delete=False) as tmp:
tmp.write(content)
# obj_list.append(Video(filename=tmp.name))
# obj_list.append(tmp.name)
key = tuple(c["_id"].values())
obj_dict.update({key: Video.from_file(tmp.name)})
print('.',end='')
In [151]:
for key, obj in obj_dict.items():
print(key)
display(obj)
NPT equilibration analysis¶
Datasets in step¶
In [152]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsNPTEquilibration:ProcessAnalyzeAndVisualize:push_filepad'
}
In [153]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'dataset': '$metadata.step_specific.dtool_push.remote_dataset',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id']['dataset'], **c} for c in cursor]
columns = ['uuid', 'name', 'uri', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [154]:
res_df
Out[154]:
Overview on objects in step¶
In [155]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsNPTEquilibration:ProcessAnalyzeAndVisualize:push_filepad'
}
In [156]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[156]:
In [157]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [158]:
res_df
Out[158]:
Global observables¶
In [159]:
query = {
"metadata.project": project_id,
'metadata.step': 'GromacsNPTEquilibration:ProcessAnalyzeAndVisualize:push_filepad',
"metadata.type": 'energy_file',
}
fp.filepad.count_documents(query)
Out[159]:
In [160]:
parameter_dict = {'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [161]:
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
In [162]:
[ c for c in cursor]
Out[162]:
In [163]:
res_list = []
cursor = fp.filepad.aggregate(aggregation_pipeline)
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
with tempfile.NamedTemporaryFile(suffix='.edr') as tmp:
tmp.write(content)
res_df = panedr.edr_to_df(tmp.name)
mi = pd.MultiIndex.from_product(
[c["_id"].values(),res_df.index],
names=[*c["_id"].keys(),'step'])
res_mi_df = res_df.set_index(mi)
res_list.append(res_mi_df)
print('.',end='')
print('')
res_df_mi = pd.concat(res_list)
res_df = res_df_mi.reset_index()
In [164]:
res_df.columns
Out[164]:
In [165]:
res_df_mi
Out[165]:
In [166]:
#n = len(res_df['nmolecules'].unique())
y_quantities = [
'Temperature',
'Pressure',
'Volume',
'Potential',
'Bond',
'Coulomb (SR)',
'Coul. recip.',
]
n = len(y_quantities)
cols = 2
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
for key, grp in res_df.groupby(['nmolecules']):
for y_quantity, position in zip(y_quantities, positions):
grp.plot('Time',y_quantity,ax=ax[position],label=key,title=y_quantity)
fig.tight_layout()
Visualize trajectory¶
In [167]:
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsNPTEquilibration:ProcessAnalyzeAndVisualize:push_filepad',
'metadata.type': 'mp4_file',
}
In [168]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[168]:
In [169]:
parameter_dict = {'nmolecules': 'metadata.system.surfactant.nmolecules'}
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_dict = {}
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
# print(metadata['metadata'])
with tempfile.NamedTemporaryFile(suffix='.mp4',delete=False) as tmp:
tmp.write(content)
# obj_list.append(Video(filename=tmp.name))
# obj_list.append(tmp.name)
key = tuple(c["_id"].values())
obj_dict.update({key: Video.from_file(tmp.name)})
print('.',end='')
In [170]:
for key, obj in obj_dict.items():
print(key)
display(obj)
In [ ]:
Pre-evaluated RDF¶
Overview¶
In [171]:
query = {
"metadata.project": project_id,
"metadata.type": {'$regex': '.*rdf$'},
"metadata.step": "GromacsNPTEquilibration:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[171]:
In [172]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [173]:
res_df
Out[173]:
Substrate - surfactant head RDF¶
In [174]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [175]:
query = {
"metadata.project": project_id,
"metadata.type": 'substrate_surfactant_head_rdf',
"metadata.step": "GromacsNPTEquilibration:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[175]:
In [176]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
aggregation_pipeline = [ match_aggregation, sort_aggregation, group_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [177]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Substrate - surfactant tail RDF¶
In [178]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [179]:
query = {
"metadata.project": project_id,
"metadata.type": 'substrate_surfactant_tail_rdf',
"metadata.step": "GromacsNPTEquilibration:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[179]:
In [180]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [181]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Surfactant head - surfactant tail RDF¶
In [182]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [183]:
query = {
"metadata.project": project_id,
"metadata.type": 'surfactant_head_surfactant_tail_rdf',
"metadata.step": "GromacsNPTEquilibration:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[183]:
In [184]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [185]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Relaxation analysis¶
Datasets in step¶
In [186]:
# queries to the data base are simple dictionaries
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad'
}
In [187]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'dataset': '$metadata.step_specific.dtool_push.remote_dataset',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id']['dataset'], **c} for c in cursor]
columns = ['uuid', 'name', 'uri', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [188]:
res_df
Out[188]:
Overview on objects in step¶
In [189]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[189]:
In [190]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
'name': '$metadata.name',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'name', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [191]:
res_df
Out[191]:
Global observables¶
In [192]:
query = {
"metadata.project": project_id,
'metadata.step': 'GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad',
"metadata.type": 'energy_file',
}
fp.filepad.count_documents(query)
Out[192]:
In [193]:
parameter_dict = {'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [194]:
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
In [195]:
[ c for c in cursor]
Out[195]:
In [196]:
res_list = []
cursor = fp.filepad.aggregate(aggregation_pipeline)
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
with tempfile.NamedTemporaryFile(suffix='.edr') as tmp:
tmp.write(content)
res_df = panedr.edr_to_df(tmp.name)
mi = pd.MultiIndex.from_product(
[c["_id"].values(),res_df.index],
names=[*c["_id"].keys(),'step'])
res_mi_df = res_df.set_index(mi)
res_list.append(res_mi_df)
print('.',end='')
print('')
res_df_mi = pd.concat(res_list)
res_df = res_df_mi.reset_index()
In [197]:
res_df.columns
Out[197]:
In [198]:
res_df_mi
Out[198]:
In [199]:
#n = len(res_df['nmolecules'].unique())
y_quantities = [
'Temperature',
'Pressure',
'Volume',
'Potential',
'Bond',
'Coulomb (SR)',
'Coul. recip.',
]
n = len(y_quantities)
cols = 2
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
for key, grp in res_df.groupby(['nmolecules']):
for y_quantity, position in zip(y_quantities, positions):
grp.plot('Time',y_quantity,ax=ax[position],label=key,title=y_quantity)
fig.tight_layout()
Visualize trajectory¶
In [200]:
query = {
'metadata.project': project_id,
'metadata.step': 'GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad',
'metadata.type': 'mp4_file',
}
In [201]:
# use underlying MongoDB functionality to check total number of documents matching query
fp.filepad.count_documents(query)
Out[201]:
In [202]:
parameter_dict = {'nmolecules': 'metadata.system.surfactant.nmolecules'}
aggregation_pipeline = [
{
"$match": query
},
{
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
},
{
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
},
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
obj_dict = {}
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
# print(metadata['metadata'])
with tempfile.NamedTemporaryFile(suffix='.mp4',delete=False) as tmp:
tmp.write(content)
# obj_list.append(Video(filename=tmp.name))
# obj_list.append(tmp.name)
key = tuple(c["_id"].values())
obj_dict.update({key: Video.from_file(tmp.name)})
print('.',end='')
In [203]:
for key, obj in obj_dict.items():
print(key)
display(obj)
Pre-evaluated RDF¶
Overview¶
In [204]:
query = {
"metadata.project": project_id,
"metadata.type": {'$regex': '.*rdf$'},
"metadata.step": "GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[204]:
In [205]:
# check files degenerate by 'metadata.type' ad 'metadata.name'
aggregation_pipeline = [
{
"$match": query
},
{ # group by unique project id
"$group": {
"_id": {
'type': '$metadata.type',
},
"object_count": {"$sum": 1}, # count matching data sets
"earliest": {'$min': '$metadata.datetime' },
"latest": {'$max': '$metadata.datetime' },
},
},
{ # sort by earliest date, descending
"$sort": {
"earliest": pymongo.DESCENDING,
}
}
]
cursor = fp.filepad.aggregate(aggregation_pipeline)
res = [ {**c['_id'], **c} for c in cursor]
columns = ['type', 'earliest', 'latest', 'object_count', '_id']
res_df = pd.DataFrame(data=res, columns=columns) # pandas Dataframe is just nice for printing in notebook
del res_df["_id"]
In [206]:
res_df
Out[206]:
Substrate - surfactant head RDF¶
In [207]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [208]:
query = {
"metadata.project": project_id,
"metadata.type": 'substrate_surfactant_head_rdf',
"metadata.step": "GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[208]:
In [209]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
aggregation_pipeline = [ match_aggregation, sort_aggregation, group_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [210]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Substrate - surfactant tail RDF¶
In [211]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [212]:
query = {
"metadata.project": project_id,
"metadata.type": 'substrate_surfactant_tail_rdf',
"metadata.step": "GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[212]:
In [213]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [214]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Surfactant head - surfactant tail RDF¶
In [215]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [216]:
query = {
"metadata.project": project_id,
"metadata.type": 'surfactant_head_surfactant_tail_rdf',
"metadata.step": "GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[216]:
In [217]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [218]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Surfactant head - counterion RDF¶
In [219]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [220]:
query = {
"metadata.project": project_id,
"metadata.type": 'counterion_surfactant_head_rdf',
"metadata.step": "GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[220]:
In [221]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [222]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'],data['rdf'][0], label='First frame RDF')
ax[pos].plot(data['dist'],data['rdf'][len(data)//2],label='Intermediate frame RDF')
ax[pos].plot(data['dist'],data['rdf'][-1],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Surfactant head - surfactant head RDF¶
In [223]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [224]:
query = {
"metadata.project": project_id,
"metadata.type": 'surfactant_head_surfactant_head_rdf',
"metadata.step": "GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[224]:
In [225]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [226]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
cutoff_index_inner = 1
cutoff_index_outer = -1
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'][cutoff_index_inner:],data['rdf'][0][cutoff_index_inner:], label='First frame RDF')
ax[pos].plot(data['dist'][cutoff_index_inner:],data['rdf'][len(data)//2][cutoff_index_inner:],label='Intermediate frame RDF')
ax[pos].plot(data['dist'][cutoff_index_inner:],data['rdf'][-1][cutoff_index_inner:],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Surfactant tail - surfactant tail RDF¶
In [227]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [228]:
query = {
"metadata.project": project_id,
"metadata.type": 'surfactant_tail_surfactant_tail_rdf',
"metadata.step": "GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[228]:
In [229]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [230]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
cutoff_index_inner = 1
cutoff_index_outer = -1
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'][cutoff_index_inner:cutoff_index_outer],data['rdf'][0][cutoff_index_inner:cutoff_index_outer], label='First frame RDF')
ax[pos].plot(data['dist'][cutoff_index_inner:cutoff_index_outer],data['rdf'][len(data)//2][cutoff_index_inner:cutoff_index_outer],label='Intermediate frame RDF')
ax[pos].plot(data['dist'][cutoff_index_inner:cutoff_index_outer],data['rdf'][-1][cutoff_index_inner:cutoff_index_outer],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
Substrate - substrate RDF¶
In [231]:
parameter_dict = {
'nmolecules': 'metadata.system.surfactant.nmolecules'}
In [232]:
query = {
"metadata.project": project_id,
"metadata.type": 'substrate_substrate_rdf',
"metadata.step": "GromacsRelaxation:ProcessAnalyzeAndVisualize:push_filepad",
}
fp.filepad.count_documents(query)
Out[232]:
In [233]:
res_dict = {}
failed_list = []
match_aggregation = {
"$match": query
}
sort_aggregation = {
"$sort": {
"metadata.datetime": pymongo.DESCENDING,
}
}
group_aggregation = {
"$group": {
"_id": { field: '${}'.format(key) for field, key in parameter_dict.items() },
"degeneracy": {"$sum": 1}, # number matching data sets
"latest": {"$first": "$gfs_id"} # unique gridfs id of file
}
}
second_sort_aggregation = {
"$sort": {
"_id.nmolecules": pymongo.DESCENDING,
}
}
aggregation_pipeline = [
match_aggregation, sort_aggregation, group_aggregation, second_sort_aggregation ]
cursor = fp.filepad.aggregate(aggregation_pipeline)
# res_list = []
for i, c in enumerate(cursor):
content, metadata = fp.get_file_by_id(c["latest"])
nmolecules = int(c["_id"]["nmolecules"])
data_str = io.StringIO(content.decode())
data = np.loadtxt(data_str, comments='#')
d = data[0] # distance bins
rdf = data[1:]
res_dict[nmolecules] = {'dist': d, 'rdf': rdf}
# res_list.append(data)
print('.',end='')
print('')
In [234]:
n = len(res_dict)
cols = 2 if n > 1 else 1
rows = round(n/cols)
cutoff_index_inner = 1
cutoff_index_outer = -1
if rows > 1:
positions = [(i,j) for i in range(rows) for j in range(cols)][:n]
else:
positions = [i for i in range(cols)][:n]
fig, ax = plt.subplots(rows,cols,figsize=(5*cols,4*rows))
if not isinstance(ax, Iterable):
ax = [ax]
# for key, grp in res_df.groupby(['nmolecules']):
for pos, (nmolecules, data) in zip(positions, res_dict.items()):
ax[pos].plot(data['dist'][cutoff_index_inner:cutoff_index_outer],data['rdf'][0][cutoff_index_inner:cutoff_index_outer], label='First frame RDF')
ax[pos].plot(data['dist'][cutoff_index_inner:cutoff_index_outer],data['rdf'][len(data)//2][cutoff_index_inner:cutoff_index_outer],label='Intermediate frame RDF')
ax[pos].plot(data['dist'][cutoff_index_inner:cutoff_index_outer],data['rdf'][-1][cutoff_index_inner:cutoff_index_outer],label='Last frame RDF')
ax[pos].set_title(nmolecules)
ax[pos].legend()
fig.tight_layout()
# fig.legend()
fig.show()
In [ ]: